import requests
from lxml import etree
import DBUtil
from concurrent.futures import ThreadPoolExecutor
import Sha1Util
import time
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
}
def xiangqing():
    '''
    爬取每个二手房的详细信息
    :return:
    '''
    sql = "select url,name from `biaoti` "
    result = DBUtil.selectTable(sql)
    for i in result:
        url =i[0]
        wangzhan = requests.get(url=url, headers=headers)
        cun = wangzhan.text
        tree = etree.HTML(cun)
        huxing=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[1]/text()')
        huxing = huxing[0] if huxing else ''
        louceng=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[2]/text()')
        louceng = louceng[0] if louceng else ''
        mianji=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[3]/text()')
        mianji = mianji[0] if mianji else ''
        hx_jiegou=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[4]/text()')
        hx_jiegou = hx_jiegou[0] if hx_jiegou else ''
        taonei=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[5]/text()')
        taonei = taonei[0] if taonei else ''
        leixing=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[6]/text()')
        leixing = leixing[0] if leixing else ''
        jz_chaoxiang=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[7]/text()')
        jz_chaoxiang = jz_chaoxiang[0] if jz_chaoxiang else ''
        jiegou=tree.xpath   ('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[8]/text()')
        jiegou = jiegou[0] if jiegou else ''
        zhuangxiu=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[9]/text()')
        zhuangxiu = zhuangxiu[0] if zhuangxiu else ''
        bili=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[10]/text()')
        bili = bili[0] if bili else ''
        dianti=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[11]/text()')
        dianti = dianti[0] if dianti else ''
        yongshui=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[12]/text()')
        yongshui = yongshui[0] if yongshui else ''
        yongdian=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[13]/text()')
        yongdian = yongdian[0] if yongdian else ''
        ranqi=tree.xpath('//*[@id="introduction"]/div/div/div[1]/div[2]/ul/li[14]/text()')
        ranqi = ranqi[0] if ranqi else ''
        maidian=tree.xpath('/html/body/div[7]/div[1]/div[2]/div/div[2]/div[2]/text()')
        maidian = maidian[0] if maidian else ''
        hx_jieshao = tree.xpath('/html/body/div[7]/div[1]/div[2]/div/div[3]/div[2]/text()')
        hx_jieshao = hx_jieshao[0] if hx_jieshao else ''
        zhoubian = tree.xpath('/html/body/div[7]/div[1]/div[2]/div/div[4]/div[2]/text()')
        zhoubian = zhoubian[0] if zhoubian else ''
        chuxing = tree.xpath('/html/body/div[7]/div[1]/div[2]/div/div[5]/div[2]/text()')
        chuxing = chuxing[0] if chuxing else ''
        print(url)
        id = Sha1Util.jia_mi(url)
        sql = '''INSERT into `xiangqing` (id,房屋户型,所在楼层,建筑面积,户型结构,套内面积,建筑类型,房屋朝向,建筑结构,装修情况,梯户比例,配备电梯,用水类型,用电类型,燃气价格,核心卖点,小区介绍,周边配套,交通出行) 
                VALUES ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')
                                                       ''' % (id,huxing, louceng, mianji, hx_jiegou, taonei, leixing, jz_chaoxiang,jiegou,zhuangxiu,bili,
                                                              dianti,yongshui,yongdian,ranqi,maidian,hx_jieshao,zhoubian,chuxing)
        DBUtil.updateTable(sql)
    # time.sleep(0.2)

def duoxiancheng():
    try:
        with ThreadPoolExecutor(20) as th:
            th.submit(xiangqing())
            # time.sleep(0.1)
    except Exception as e:
        print(e)


duoxiancheng()
# 22:47  569
# 23:04  1793